# -*- coding: utf-8 -*-
#
# Copyright (c) 2011,2012 RMIT e-Research Office
#   (RMIT University, Australia)
# All rights reserved.
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#    *  Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#    *  Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#    *  Neither the name of RMIT University nor the
#       names of its contributors may be used to endorse or promote products
#       derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

"""
.. module:: Afirc
   :synopsis: Ingests records form the AFIRC respository into SMRA

.. moduleauthor:: Ian Thomas <Ian.Edward.Thomas@rmit.edu.au>


"""

from django.conf import  settings
import suds
import logging
import urllib2
import datetime
import time

from smra.smra_portal.models import Schema
from smra.smra_portal.models import System
from smra.smra_portal.models import Repository
from smra.smra_portal.models import ParameterName
from smra.smra_portal.ReposPlugin import Error
from smra.smra_portal.ReposPlugin import InvalidMediaObject
from smra.smra_portal.ReposPlugin import InvalidConfigurationException 
from smra.smra_portal.ReposPlugin import MissingIDException
from smra.smra_portal.ReposPlugin import Timer
from django.utils.http import urlquote_plus

logger = logging.getLogger(__name__)


class ReposConverter(object):
    """ 
    Converts the data on the AFI website into form that can be added 
    to models
    """

    base_field_trans = {'': {
                            'Title':'Title',
                            'Subtitle':'Subtitle',
                            'Subject':'Subject',
                            'Summary':'Summary',
                            'Responsibility':'Responsibility',
                            'ISBN':'ISBN_ISSN',
                            'Author':'Author',
                            'PubDate':'Publication Date',
                            'Publisher':'Publisher',
                            'PhysDes':'Physical Description',
                            'Source':'Source',                        
                            'Collection':'AFI Research',                            
                            'Record_Type':'Unknown'
                            },                                                     
                   'Index':{
                            'Title':'Title',
                            'Subject':'Subject',
                            'ISSN':'ISBN_ISSN',
                            'Source':'Source',                            
                            'Collection':'AFI Research',
                            'Record_Type':'Index'
                             },
                    'Bibliography':{
                            'Title':'Title',
                            'Subject':'Subject',
                            'Responsibility':'Responsibility',
                            'PubDate':'Publication Date',
                            'Collection':'AFI Research',                                       
                            'Record_Type':'Bibliography'},
                        
                    'DVD Research Folder':{
                            'Title':'Title',
                            'Subtitle':'Subtitle',
                            'Subject':'Subject',
                            'Summary':'Summary',                           
                            'Responsibility':'Responsibility',       
                            'Author':'Author',
                            'PubDate':'Publication Date',
                            'Publisher':'Publisher',
                            'PhysDes':'Physical Description',
                            'Source':'Source',
                            'Collection':'AFI Research',
                            'Record_Type':'DVD Folder',
                           },
                        
                    'Script':{
                            'Title':'Title',
                            'Subtitle':'Subtitle',
                            'Subject':'Subject',
                            'Summary':'Summary',                           
                            'Responsibility':'Responsibility', 
                            'Author':'Author',
                            'PubDate':'Publication Date',
                            'Publisher':'Publisher',
                            'Contents':'Contents',
                            'PhysDes':'Physical Description',
                            'Source':'Source',                        
                            'Collection':'AFI Research',                            
                            'Record_Type':'Script'
                           },
                        
                    'CD-ROM':{'Title':'Title',
                           'Subtitle':'Subtitle',
                           'Summary':'Summary',  
                           'ISSN':'ISBN_ISSN',
                           'PubDate':'Publication Date',
                           'Publisher':'Publisher',
                           'PhysDes':'Physical Description',
                           'Source':'Source',                        
                           'Collection':'AFI Research',                            
                           'Record_Type':'CD-ROM'                        
                           },
                    'Digital image:JPEG':{
                           'Title':'Title',
                           'Subject':'Subject',
                           'Summary':'Summary',
                           'Source':'Source',    
                           'Collection':'AFI Research',                            
                           'Record_Type':'Digital Image'       
                          },
                    'Monograph - old collection':{
                           'Title':'Title',
                           'Subtitle':'Subtitle',
                           'Subject':'Subject',
                           'Summary':'Summary',                           
                           'Responsibility':'Responsibility',    
                           'ISSN':'ISBN_ISSN',
                           'Author':'Author',
                           'PubDate':'Publication Date',
                           'Publisher':'Publisher',
                           'Contents':'Contents',
                           'PhysDes':'Physical Description',
                           'Source':'Source',      
                           'Collection':'AFI Research',                            
                           'Record_Type':'Monograph'
                          },    

                    'Monograph - yet to catalogue':{
                           'Title':'Title',
                           'Subtitle':'Subtitle',
                           'Subject':'Subject',
                           'Summary':'Summary',                           
                           'Responsibility':'Responsibility',   
                           'ISBN':'ISBN_ISSN',
                           'Author':'Author',
                           'PubDate':'Publication Date',
                           'Publisher':'Publisher',
                           'Contents':'Contents',
                           'PhysDes':'Physical Description',
                           'Source':'Source',      
                           'Collection':'AFI Research',                            
                           'Record_Type':'Monograph'
                           },
                        
                    'Monograph -':{
                           'Title':'Title',
                           'Subject':'Subject',
                           'Summary':'Summary',                           
                           'Responsibility':'Responsibility',   
                           'ISBN':'ISBN_ISSN',
                           'Author':'Author',
                           'PubDate':'Publication Date',
                           'Publisher':'Publisher',
                           'PhysDes':'Physical Description',
                           'Source':'Source',      
                           'Collection':'AFI Research',                            
                           'Record_Type':'Monograph'
                           },
                        
                    'Monograph - yet to accession':{
                           'Title':'Title',
                           'Subtitle':'Subtitle',
                           'Subject':'Subject',  
                           'Responsibility':'Responsibility',   
                           'ISBN':'ISBN_ISSN',
                           'Author':'Author',
                           'PubDate':'Publication Date',
                           'Publisher':'Publisher',  
                           'PhysDes':'Physical Description',   
                           'Collection':'AFI Research',                            
                           'Record_Type':'Monograph'
                         },
                        
                    'Monograph ?':{
                           'Title':'Title',
                           'Subtitle':'Subtitle',
                           'Subject':'Subject',  
                           'Responsibility':'Responsibility',   
                           'ISBN':'ISBN_ISSN',
                           'Author':'Author',
                           'PubDate':'Publication Date',
                           'Publisher':'Publisher',  
                           'PhysDes':'Physical Description',   
                           'Source':'Source',      
                           'Collection':'AFI Research',                            
                           'Record_Type':'Monograph'
                            },
                        
                    'Monograph':{ 
                            'Title':'Title',
                            'Subtitle':'Subtitle',
                            'Subject':'Subject',
                            'Summary':'Summary',
                            'Responsibility':'Responsibility',
                            'ISBN':'ISBN_ISSN',
                            'Author':'Author',
                            'PubDate':'Publication Date',
                            'Publisher':'Publisher',
                            'PhysDes':'Physical Description',
                            'Source':'Source',                        
                            'Collection':'AFI Research',                            
                            'Record_Type':'Monograph'
                            },
                    'Monograph - Copy':{
                            'Title':'Title',
                            'Subtitle':'Subtitle',
                            'Subject':'Subject',
                            'Responsibility':'Responsibility',
                            'ISBN':'ISBN_ISSN',
                            'Author':'Author',
                            'PubDate':'Publication Date',
                            'Publisher':'Publisher',
                            'PhysDes':'Physical Description',
                            'Source':'Source',                        
                            'Collection':'AFI Research',                            
                            'Record_Type':'Monograph'
                            },
                        
                    'Monograph - Levy Collection':{
                            'Title':'Title',
                            'Subtitle':'Subtitle',
                            'Subject':'Subject',
                            'Summary':'Summary',
                            'Responsibility':'Responsibility',
                            'ISBN or ISSN':'ISBN_ISSN',
                            'Author':'Author',
                            'PubDate':'Publication Date',
                            'Publisher':'Publisher',
                            'Contents':'Contents',
                            'PhysDes':'Physical Description',
                            'Source':'Source',                        
                            'Collection':'Levy',                            
                            'Record_Type':'Monograph'},
                        
                    'Folder':{
                            'Title':'Title',
                            'Subtitle':'Subtitle',
                            'Subject':'Subject',
                            'Responsibility':'Responsibility',
                            'Collection':'AFI Research',                            
                            'Record_Type':'Folder'
                            },
                    'Newspaper Article':{
                             'Title':'Title',
                            'Subtitle':'Subtitle',
                            'Subject':'Subject',
                            'Summary':'Summary', 
                            'Author':'Author',
                            'Date':'Publication Date',  
                            'Collection':'AFI Research',                            
                            'Record_Type':'Article'
                    },
                    'Clippings file':{
                                      
                            'Title':'Title',
                            'Subtitle':'Subtitle',
                            'Subject':'Subject',
                            'Summary':'Summary',
                             'ISSN':'ISBN_ISSN',
                               'PubDate':'Publication Date',
                            'Publisher':'Publisher',
                            'Contents':'Contents',
                            'PhysDes':'Physical Description',
                            'Source':'Source',                        
                            'Collection':'AFI Research',                            
                            'Record_Type':'Clippings'
                                      },
                    'Poster':{
                            'Title':'Title',
                            'Subtitle':'Subtitle',
                            'Subject':'Subject',
                            'Summary':'Summary',
                            'Responsibility':'Responsibility',
                            'Author':'Author',
                            'PubDate':'Publication Date',
                            'Publisher':'Publisher',
                            'PhysDes':'Physical Description',
                            'Collection':'AFI Research',                            
                            'Record_Type':'Poster'
                        
                            },
                    'Journal':{
                               'Title':'Title',
                            'Subtitle':'Subtitle',
                            'Subject':'Subject',
                            'Summary':'Summary',
                            'Author':'Author',
                            'Date':'Publication Date',  
                               'Collection':'AFI Research',                            
                            'Record_Type':'Article'
                            },
                    'Journal article':{
                                       'Title':'Title',
                            'Subtitle':'Subtitle',
                            'Subject':'Subject',
                            'Summary':'Summary',
                            'Responsibility':'Responsibility',
                               'Author':'Author',
                            'PubDate':'Publication Date',
                             'PhysDes':'Physical Description',
                             'Collection':'AFI Research',                            
                            'Record_Type':'Article'
                                       },
                        
                    'Microfilm':{
                                 'Title':'Title',
                                 'ISSN':'ISBN_ISSN',
                                 'PubDate':'Publication Date',
                                 'Publisher':'Publisher',
                                 'PhysDes':'Physical Description',
                            'Source':'Source',                        
                            'Collection':'AFI Research',                            
                            'Record_Type':'Microfilm'

                                 },
                    'Special issue':{
                                      'Title':'Title',
                                      'Subject':'Subject',
                                       'PubDate':'Publication Date',
                                           'Publisher':'Publisher',
                                             'Collection':'AFI Research',                            
                                             'Record_Type':'Microfilm'
                                     },
                    'Annual report':{
                                      'Title':'Title',
                                      'Subject':'Subject',
                                         'Responsibility':'Responsibility',
                            'ISBN or ISSN':'ISBN_ISSN',
                                'PubDate':'Publication Date',
                                 'Publisher':'Publisher',
                                 'PhysDes':'Physical Description',

                            'Source':'Source',                        
                            'Collection':'AFI Research',                            
                             'Record_Type':'Annual Report'
                                     
                                     },
                    'Music LP':{
                            'Title':'Title',
                            'Subtitle':'Subtitle',
                            'Subject':'Subject',
                            'Summary':'Summary',
                                    'Responsibility':'Responsibility',
                               'Author':'Author',
                                'PubDate':'Publication Date',
                                           'Publisher':'Publisher',
                                             'Contents':'Contents',
                            'PhysDes':'Physical Description',
                            'Source':'Source',                        
                            'Collection':'AFI Research',                            
                            'Record_Type':'Music LP'
                                },
                    'Festival catalogue':{
                                           'Title':'Title',
                            'Subtitle':'Subtitle',
                            'Subject':'Subject',
                            'Summary':'Summary',
                                   'Responsibility':'Responsibility',
                                     'ISBN or ISSN':'ISBN_ISSN',
                                      'Author':'Author',
                                'PubDate':'Publication Date',
                                           'Publisher':'Publisher',
                                             'Contents':'Contents',
                            'PhysDes':'Physical Description',
                            'Source':'Source',                        
                            'Collection':'AFI Research',                            
                            'Record_Type':'Festival Catalogue'
                         
                                          },
                    'Microform':{
                                  'Title':'Title',
                                    'Subject':'Subject',
                                   'ISBN or ISSN':'ISBN_ISSN',
                                    'PubDate':'Publication Date',
                                           'Publisher':'Publisher',
                                'PhysDes':'Physical Description',
                            'Source':'Source',                        
                            'Collection':'AFI Research',                            
                            'Record_Type':'Microfilm'


                                 },
                    'Serial - Levy Collection':{
                                                  'Title':'Title',
                                                   'PubDate':'Publication Date',
                                           'Publisher':'Publisher',
                                            'Source':'Source',                        
                            'Collection':'Levy',                            
                            'Record_Type':'Serial'
                                                  
                                                },
                    'Webpage':{
                               'Title':'Title',
                            'Subtitle':'Subtitle',
                            'Subject':'Subject',
                            'Summary':'Summary',
                                'Author':'Author',
                            'Date':'Publication Date',
                                'Collection':'AFI Research',                            
                            'Record_Type':'Webpage'

                               },
                    'Video tape':{
                              'Title':'Title',
                            'Summary':'Summary',
                            'Collection':'AFI Research',                            
                            'Record_Type':'Video Tape'
                                  },
                    'Interim':{
                            'Title':'Title',
                            'Subtitle':'Subtitle',
                            'Subject':'Subject',
                            'Summary':'Summary',
                                   'Responsibility':'Responsibility',
                                     'ISBN or ISSN':'ISBN_ISSN',
                                      'Author':'Author',
                                'PubDate':'Publication Date',
                                           'Publisher':'Publisher',
                                             'Contents':'Contents',
                            'PhysDes':'Physical Description',
                            'Source':'Source',                        
                            'Collection':'AFI Research',                            
                            'Record_Type':'Monograph'
                               },
                        
                    'Script - Crawford Productions':{
                                   'Title':'Title',
                            'Subtitle':'Subtitle',
                            'Subject':'Subject',
                          'Responsibility':'Responsibility',
                              'Author':'Author',
                                 'PubDate':'Publication Date',
                            'ISBN or ISSN':'ISBN_ISSN',
                                         'Publisher':'Publisher',
                         'PhysDes':'Physical Description',
                          'Collection':'Crawford', 
                          'Record_Type':'Script'
 
                                                     },
                    'Serial':{
                              'Title':'Title',
                            'Subtitle':'Subtitle',
                            'Subject':'Subject',
                            'Summary':'Summary',
                                   'Responsibility':'Responsibility',
                                     'ISSN':'ISBN_ISSN',
                                      'Author':'Author',
                                'PubDate':'Publication Date',
                                           'Publisher':'Publisher',
                                           
                                'PhysDes':'Physical Description',
                            'Source':'Source',                        
                            'Collection':'AFI Research',                            
                            'Record_Type':'Serial'            
                                           
                              },
                    'Still':{
                                  'Title':'Title',
                              'Subject':'Subject',
                            'Summary':'Summary',
                                   'Responsibility':'Responsibility',
                                       'PubDate':'Publication Date',
                                           'Publisher':'Publisher',
                                              'PhysDes':'Physical Description',
                            'Source':'Source',                        
                            'Collection':'AFI Research',                            
                            'Record_Type':'Still'         
                             },
                    'DVD':{
                            'Title':'Title',
                            'Subtitle':'Subtitle',
                            'Subject':'Subject',
                            'Summary':'Summary',
                                   'Responsibility':'Responsibility',
                                     'ISBN':'ISBN_ISSN',
                                     'PubDate':'Publication Date',
                                           'Publisher':'Publisher',
                                           
                                            'Contents':'Contents',
                            'PhysDes':'Physical Description',
                            'Source':'Source',                        
                            'Collection':'AFI Research',                            
                            'Record_Type':'Still'         
                                             
                           }
                    }
                    
    ext_field_trans = {'': {'Issue_number':'Issue',
                            'Journal_title':'Journal',
                            'Page_number':'Page',
                            'Volume_number':'Volume',
                            'Donation':'Donation'},
                       'Index':{},                    
                       'Bibliography':{},
                       'DVD Research Folder':{'Journal_title':'Issue',
                                              'Newspaper_Title':'Journal',
                                              'Page_number':'Page',
                                              'Volume_number':'Volume'},
                       'Script':{'Donation':'Donation'},
                    'CD-ROM':{},
                    'Digital image:JPEG':{},
                    'Monograph - old collection':{
                                                  'Donation':'Donation',
                                                  'Language':'Language'
                                                    },
                    'Monograph - yet to catalogue':{'Donation':'Donation',
                                                    'Language':'Language'},
                    'Monograph -':{'Donation':'Donation'},
                    'Monograph - yet to accession':{},
                    'Monograph ?':{},
                    'Monograph':{
                                 'Donation':'Donation',
                                                  'Language':'Language'
                                 },
                    'Monograph - Copy':{
                                        'Donation':'Donation',
                                        },
                    'Monograph - Levy Collection':{'Donation':'Donation',
                                                  'Language':'Language'},
                    'Folder':{},
                    'Newspaper Article':{'Page_number':'Page',
                                         'Newspaper_Title':'Journal'},
                    'Clippings file':{
                                       'Donation':'Donation',
                                                  'Language':'Language'
                                      },
                    'Poster':{
                              'Issue_number':'Issue',
                            'Journal_title':'Journal',
                            'Page_number':'Page',
                            'Volume_number':'Volume',
                                    'Language':'Language'

                              },
                    'Journal':{  'Issue_number':'Issue',
                            'Journal_title':'Journal',
                            'Page_number':'Page',
                            'Volume_number':'Volume'},
                    'Journal article':{  'Issue_number':'Issue',
                            'Journal_title':'Journal',
                            'Page_number':'Page',
                            'Volume_number':'Volume'},
                    'Microfilm':{},
                    'Special issue':{ 'Language':'Language'
                                     },
                    'Annual report':{ 'Language':'Language'},
                    'Music LP':{'Performer':'Performer'},
                    'Festival catalogue':{ 'Donation':'Donation',
                                                  'Language':'Language'},
                    'Microform':{},
                    'Serial - Levy Collection':{
                                                 'Donation':'Donation'},
                    'Webpage':{},
                    'Video tape':{},
                    'Interim':{ 'Donation':'Donation',
                                                  'Language':'Language'},
                    'Script - Crawford Productions':{},
                    'Serial':{ 'Donation':'Donation',
                                                  'Language':'Language'},
                    'Still':{ 'Donation':'Donation'},
                    'DVD':{'Performer':'Performer',
                           'Also_Titled':'Alternate Title',
                           'Standard_Number':'Product Code',
                           'Credits':'Credits'}
                    }
                       
        
    REPOS_NAME = "AFIRC"    
    ns = "http://smra.edu.au/smra/afi"
    
    def get_keywordtrans(self,keyword_trans):
        """ Convert raw field names in db into cleaned up version in smra"""
        keyword_trans['PubDate'] = 'Publication Date'
        keyword_trans['PhysDes'] = 'Physical Description'
        keyword_trans['Record_Created'] = 'Record Created' 
        keyword_trans['When_Cataloged'] = 'When Cataloged'
        keyword_trans['Call_No'] = 'Call No'
        return keyword_trans
        
    
    def __init__(self,fake=False):
        """ 
        Specifies whether this converter will connect to webservice or fake
        the calls for testing
        """
        self.testing = fake
            
    def _convert_one_record(self,repos_plugin, attr):
        """ 
        Creates media object with metadata 
        """
        try:
            ident = attr['ID']      
        except KeyError:
            raise MissingIDException("AFI record missing required ID field") 
        try:
            if type(attr['Title']) is list:
                # TODO: if any parts of title have commas, use semicolons instead
                desc = ", ".join(list(attr['Title']))
            else:
                desc = attr['Title']
        except KeyError:
            raise InvalidMediaObject("AFI record %s missing required Title "
                                     "field" % ident)
        try:
            record_type = attr['Record_Type']  
        except KeyError:
            raise InvalidMediaObject("AFI record %s missing required"
                                     " Record_type field" % ident) 
        if type(record_type) is list:
            record_typelist = [x.encode('ascii','ignore')
                                for x in list(record_type)]
        else:
            record_type = str(record_type).encode('ascii','ignore')
            record_typelist = []
            record_typelist.append(record_type)
        # TODO: fix unicode problems
        desc_str = desc.encode('ascii','ignore')
        logger.debug("ident=%s" % ident)
        # FIXME: what if afi record was missing or scrambled
        keyword_trans = dict([(x,x) for x in attr.keys()])
        #keyword_trans = self.get_keywordtrans(keyword_trans)
        meta = {}
        for x in attr.keys():
            if type(attr[x]) is list:
                joined_att = ', '.join(attr[x])
                if attr[x]:
                    meta[keyword_trans[x]] = str(joined_att.encode('ascii',
                                                                   'ignore'))
            else:
                if attr[x]:
                    meta[keyword_trans[x]] = str(attr[x].encode('ascii',
                                                                'ignore'))
        if logger.isEnabledFor(logging.DEBUG):
            logger.debug("meta=%s" % meta)
            
        finaltype = []    
        base_record_data = {}    
        ext_record_data = {}
            
        for record_type in record_typelist:
            
            if record_type in self.base_field_trans.keys():
                base_type_record = self.base_field_trans[record_type]
                
                if 'Collection' in base_type_record:                
                    base_record_data['Collection'] = base_type_record['Collection']
                else:
                    base_record_data['Collection'] = "Unknown"
                                 
                if 'Record_Type' in base_type_record:
                    finaltype.append(base_type_record['Record_Type'])
                else:
                    finaltype.append("Unknown")
            else:               
                finaltype.append("Unknown")
                base_type_record = {}
                #FIXME: if no matching type, then we skip but, may match on 
                #another iterations, so need to indicate this
                
            if record_type in self.ext_field_trans.keys():
                ext_type_record = self.ext_field_trans[record_type]
            else:
                ext_type_record = {}
        
                
            logger.debug("base_type_record=%s" % base_type_record)
            logger.debug("ext_type_record=%s" % ext_type_record)
    
            isbn_value = None
            issn_value = None
            for m in meta:
                
                if m == 'ISBN':
                    isbn_value = meta[m]
                    continue
                
                if m == 'ISSN':
                    issn_value = meta[m]
                    continue
                
                if m in base_type_record: 
                    base_record_data[base_type_record[m]] = meta[m]
                else:
                    pass                
                if m in ext_type_record: 
                    ext_record_data[ext_type_record[m]] = meta[m]
                else:
                    pass
                                
            if 'ISBN or ISSN' in base_type_record:
                res = []                
                if isbn_value:
                    res.append(isbn_value)
                if issn_value:                    
                    res.append(issn_value)
                base_record_data[base_type_record['ISBN or ISSN']] = "/".join(res)
                logger.debug("ISBN/ISSN resolution")                                    
            else:
                for key in ('ISBN','ISSN'):
                    logger.debug("trying %s" % key)
                    if key in base_type_record and key in meta:
                        base_record_data[base_type_record[key]] = meta[key]
                        logger.debug("defaultcase %s" % key)                                    
                    else:
                        logger.debug("no match on %s" % key)
        
                
        
        logger.debug("finaltype=%s" % finaltype)
        logger.debug("base_record_data=%s" % base_record_data)        
        logger.debug("ext_record_data=%s" % ext_record_data)
        
        #FIXME: this should be base_record_data?
        if not base_type_record:
            return []
            
        
        d = {}
        for x in finaltype: 
            d[x]=x
        base_record_data['Type'] = ', '.join(d.values())    
   
        final_datas = []
     
   
        final_data = []
        final_data.append(("http://rmit.edu.au/schemas/smra",'base'))
        final_data.append(base_record_data)
        final_datas.append(final_data)
        
   
        if ext_record_data:
            final_data = []
            final_data.append(("http://rmit.edu.au/schemas/smra",'extended'))
            final_data.append(ext_record_data)
            final_datas.append(final_data)
        
        
        final_data = []
        final_data.append(("http://rmit.edu.au/schemas/smra",'system'))
        record_created = meta['Record_Created'] if 'Record_Created' in meta else ''
        call_no = meta['Call_No'] if 'Call_No' in meta else ''
        final_data.append({'Record Type': ', '.join(record_typelist),
                           'Created':record_created,
                           'ID':ident,
                           'Call No':call_no})
        final_datas.append(final_data)
         
         
        afi_link = settings.AFI_LINK % ident     
        logger.debug("afilink=%s" % afi_link)
     
        final_data = []
        final_data.append(("http://rmit.edu.au/schemas/smra",'external'))
        final_data.append({'IMDB': 'http://www.imdb.com/find?s=all&q=%s'
                                         % urlquote_plus(desc),
                            'Link to Original Record':afi_link})
        final_datas.append(final_data)
        
        logger.debug("final_data = %s" % final_data)
        
        obj  = repos_plugin.create_media_object(repos_name=self.REPOS_NAME,
                                               ident=ident,
                                               description=desc_str,
                                               metadata=final_datas)
        
        
        objs = []
        objs.append(obj)
        return objs
#        
# 
#        return []
#    
#        objs = []
#      
#        record_datas = []        
#        for name in record_typelist:        
#            logger.debug("slug name = %s" % name)
#            from django.template.defaultfilters import slugify 
#            slugname = slugify(str(name))        
#            logger.debug("slug name = %s" % slugname)
#            record_data = ((self.ns,slugname),meta)
#            record_datas.append(record_data)
#        
#        #timer = Timer()
#
#        obj  = repos_plugin.create_media_object(repos_name=self.REPOS_NAME,
#                                               ident=ident,
#                                               description=desc_str,
#                                               metadata=record_datas)
#        obj = repos_plugin.set_last_ingested_metadata(repos_name=self.REPOS_NAME,
#                                          obj=obj,
#                                            metadata=(
#                                          (("http://smra.edu.au/smra/ingest",
#                                           'metadata'),
#                                          {'Last Ingested':
#                                            str(datetime.date.today())}),),)
#        
#        logger.debug("afilink=%s" % afi_link)
#        obj = repos_plugin.set_last_ingested_metadata(
#                                      repos_name=self.REPOS_NAME,
#                                      obj=obj,
#                                      metadata=(
#                                        (("http://smra.edu.au/smra",
#                                                   'external-collections'),
#                                    {'Internet Media Database (www.imdb.com)':
#                                       'http://www.imdb.com/find?s=all&q=%s'
#                                         % urlquote_plus(desc),
#                                    'AFI':afi_link}),),
#                                      )
#        
#        
#        
#        
#        #logger.info("time for object creation = %s secs"
#        #                 % timer.get_elapsed_time())
#        objs.append(obj)
#        return objs
            
    def _make_web_call(self, query):
        """
        Make a AFI service call (unless fake) using the query_str
        """
        if not self.testing:
            logger.debug("making call")
            timer = Timer()
            res = self.client.service.queryX(query, "null", "null", "null")
            logger.info("time for webcall = %s secs" % timer.get_elapsed_time())
        else:
            res = self.testing[query]
            logger.debug("fake res = %s" % res)
            logger.debug("fake type(res) = %s" % type(res))
            logger.debug("fake testing = %s" % self.testing)
            logger.debug("querystr = %s" % query)
            #import pdb; pdb.set_trace() 
        return res

   
    def process_one_ws_call(self,repos_plugin, start,end):
        """ 
        Make one call to the WS and process returned records
        """
        res = None
        querystr = "Find ID=%d:%d" % (start, end)
        if logger.isEnabledFor(logging.DEBUG):            
            logger.debug("query=%s" % querystr)
        ok_web_call = False
        timer = Timer()
        for attempts in xrange(0,settings.AFI_CALL_ATTEMPTS):
            logger.debug("attempt=%s" % attempts)
            try:
                res = self._make_web_call(querystr)
            except Exception, e:
                import traceback,sys
                # FIXME: check if get partial results
                traceback.print_exc(file=sys.stdout)               
                message = "call failed for %s with exception  %s" % (querystr,e)
                logger.warning(message)
                logger.debug("sleeping")
                import time
                time.sleep(60 * (attempts+1))
            else:
                ok_web_call = True
                if attempts > 0:
                    logger.info("call %s okay" % attempts)
                break
        if not ok_web_call:
            message = "Error ingesting from AFI " \
            "in %s:%s after %s tries" % (start,end, settings.AFI_CALL_ATTEMPTS)
            logger.error(message)
            #print "res=%s" % res
            #import pdb; pdb.set_trace()  
            logger.info("time for call+ingestion = %s secs" % 
                        timer.get_elapsed_time())        
            return (None,0,0)
        else:
            message = "Run on %s:%s" % (start,end)
            logger.info(message)
            logger.debug("res=%s" % res)
            result_count = 0
            new_objs_count = 0
            new_objs = []
            for recordset in list(res.Catalog_recordset):
                if logger.isEnabledFor(logging.DEBUG):
                    logger.debug("recordset=%s" % list(recordset))
                if  type(recordset[1]) is list:
                    for record in recordset[1]:
                        attr = dict(record)
                        try:
                            objs = self._convert_one_record(repos_plugin, attr)
                            for obj in objs:
                                new_objs_count += 1
                                new_objs.append(obj)
                        except MissingIDException:
                            raise InvalidMediaObject(
                                                     "AFI record missing "
                                                     "required ID field "
                                                     "in range %s:%s "
                                                     "(%s-th record)" 
                                                     % (start,end,result_count))
                        result_count += 1                                            
                else:
                    attr = dict(recordset[1])
                    try:
                        objs = self._convert_one_record(repos_plugin, attr)
                        for obj in objs:
                            new_objs_count += 1
                            new_objs.append(obj)
                    except MissingIDException:
                        raise InvalidMediaObject(
                                                 "AFI record missing "
                                                 "required ID field "
                                                 "in range %s:%s "
                                                 "(%s-th record)" 
                                                 % (start,end,result_count))
                    result_count += 1
            logger.info("%s elements returned" % result_count)
            logger.info("%s new objs" % new_objs_count)
            logger.info("time for call+ingestion = %s secs"
                         % timer.get_elapsed_time())        
            return (new_objs, new_objs_count, result_count)
            
    def ingest(self,repos_plugin,system):
        try:
            sys = System.objects.get(name=system)
        except System.DoesNotExist():
            raise InvalidConfigurationException(
                                "Missing System definition for %s" % system)
        logger.debug("REPOS_NAME = %s" % self.REPOS_NAME)
        try:
            self.repos = Repository.objects.get(name=self.REPOS_NAME, 
                                                system=sys)
        except Repository.DoesNotExist:
            self.repos = Repository()
            self.repos.name = self.REPOS_NAME
            self.repos.system = sys
            self.repos.save()
        # This may be needed if having problems with RMIT firewall
        if not self.testing:
            if settings.PROXY:
                proxy = urllib2.ProxyHandler(settings.PROXY)
                auth = urllib2.HTTPBasicAuthHandler()
                opener = urllib2.build_opener(proxy, auth, urllib2.HTTPHandler)
                urllib2.install_opener(opener)
            try:
                self.client = suds.client.Client(settings.AFIWSDL,
                                        timeout=settings.SERVICE_CALL_TIMEOUT,
                                        prefixes=False)
            except Exception,e:
                logger.error("Cannot retrieve AFI wsdl")
                raise InvalidConfigurationException(e)
                
        else:
            logger.debug("we are testing")
        # We cannot connect to AFI and get all records at once, so break up
        # into groups and ingest each group at a time    
        endindex = settings.MAXIMUM_AFI_ID
        step = settings.AFI_CALL_STEP
        index = settings.START_AFI_ID
        new_objs_total = 0
        result_total = 0
        while index < endindex:
            (_,new_objs_count, result_count) =  self.process_one_ws_call(
                                                            repos_plugin,
                                                            index,
                                                            index+step-1)
            new_objs_total += new_objs_count
            result_total += result_count
            index += step 
        print "total elements returned = %s" % result_total 
        print "total new objs created = %s" % new_objs_total 
                    
        return "Hello from AFIRC plugin"
        
    
    def setup_schemas(self,repos_plugin):
        """ Create the required schema definition for AFI data
        """
        
        schemas = (('Title',ParameterName.STRING,90,""),
                   ('Subtitle',ParameterName.STRING,85,""),
                   ('Subject',ParameterName.STRING,80,""),
                   ('Summary',ParameterName.STRING,75,""),
                   ('Responsibility',ParameterName.STRING,70,""),
                   ('ISBN_ISSN',ParameterName.STRING,65,""), #check underscore
                   ('Author',ParameterName.STRING,60,""),
                   ('Publication Date',ParameterName.STRING,55,""),
                   ('Publisher',ParameterName.STRING,50,""),
                   ('Contents',ParameterName.STRING,45,""),
                   ('Physical Description',ParameterName.STRING,40,""),
                   ('Source',ParameterName.STRING,35,""),
                   ('Collection',ParameterName.STRING,30,""),
                   ('Type',ParameterName.STRING,25,""))
                   
        repos_plugin.create_schema(ns="http://rmit.edu.au/schemas/smra",
                            name="base",
                            desc="Details",
                            parameters=schemas)

        
        schemas = (('Issue',ParameterName.STRING,90,""),
                   ('Journal',ParameterName.STRING,85,""),
                   ('Page',ParameterName.STRING,80,""),
                   ('Volume',ParameterName.NUMERIC,75,""),
                   ('Newspaper',ParameterName.STRING,70,""),
                   ('Performer',ParameterName.STRING,65,""),
                   ('Alternate Title',ParameterName.STRING,60,""),
                   ('Product Code',ParameterName.STRING,55,""),
                   ('Credits',ParameterName.STRING,50,""),
                   ('Donation',ParameterName.STRING,45,""),
                   ('Language',ParameterName.STRING,40,""),
                   ('Comment',ParameterName.STRING,35,""),
                   ('Notes',ParameterName.STRING,30,""),
                   ('Production Year',ParameterName.YEAR,25,""),
                   ('First Release Year',ParameterName.YEAR,20,""))
                   
        repos_plugin.create_schema(ns="http://rmit.edu.au/schemas/smra",
                            name="extended",
                            desc="Extended Details",
                            parameters=schemas)

        schemas = (('Record Type',ParameterName.STRING,90,""),
                   ('Created',ParameterName.DATE,80,""),
                   ('ID',ParameterName.STRING,75,""),
                   ('Call No',ParameterName.STRING,70,""))
                   
        repos_plugin.create_schema(ns="http://rmit.edu.au/schemas/smra",
                            name="system", 
                            desc="System Details",
                            parameters=schemas)


        #TODO: this schema could be created in ReposPlugin
        schemas  = (('IMDB',
                    ParameterName.LINK,90,""),
                   ('Link to Original Record',ParameterName.LINK,80,""))
       
        repos_plugin.create_schema(ns="http://rmit.edu.au/schemas/smra",
                            name="external",
                            desc="External Collections",
                            parameters=schemas)
    
               
    def _check_object(self,attr):
        """ 
        Using data in attr dict, add media object to repos
        """
        if type(attr['Title']) is list:
            desc = ", ".join(list(attr['Title']))
        else:
            desc = attr['Title']
        # TODO: fix unicode problems
        desc_str = desc.encode('ascii','ignore')
        #desc_str = desc
        ident = attr['ID']                  
        record_type = attr['Record_Type']  
        if type(record_type) is list:
            record_typelist = [x.encode('ascii','ignore') 
                               for x in list(record_type)]
        else:
            record_typelist = []
            record_typelist.append(str(record_type).encode('ascii','ignore'))
            
        logger.debug("ident=%s" % ident)
        keyword_trans = dict([(x,x) for x in attr.keys()])
        
        meta = {}
        for x in attr.keys():
            if type(attr[x]) is list:
                att = ','.join(attr[x])
                if attr[x]:
                    meta[keyword_trans[x]] = str(att.encode('ascii','ignore'))
            else:
                if attr[x]:
                    meta[keyword_trans[x]] = str(attr[x].encode('ascii',
                                                                'ignore'))
        logger.debug("meta=%s" % meta)
        return (record_typelist,meta)        



    
    